from sklearn.metrics import mean_squared_error, mean_absolute_error, roc_auc_score, recall_score
import pandas as pd
import numpy as np

withmetadata = 1
withkeywords = 2
withgenres = 3

random_state = 6
llm_model_size = "72b"
truncate_size_300 = False
datatype = withmetadata
# model = f"Qwen/Qwen1.5-{llm_model_size}-Chat"
model = "Qwen1.5-7B-Chat-sft2"
# model = "gpt-3.5-turbo-0125"
# model = "SVD"

# 
if datatype == withkeywords:
    result_df_file = "../datasets/result/number/response/test_set_withkeywords_{}_{}_1k.csv".format(
        model.replace("/", ""), random_state)
elif datatype == withmetadata:
    result_df_file = "../datasets/result/number/response/test_set_{}_{}_1k.csv".format(model.replace("/", ""),
                                                                                       random_state)
elif datatype == withgenres:
    result_df_file = "../datasets/result/number/response/test_set_withgenres_{}_{}_1k.csv".format(
        model.replace("/", ""), random_state)
print(result_df_file)

real_df = pd.read_csv(f"../datasets/llm_pre/test_set_{random_state}_1k.csv", sep=",", low_memory=False)
result_df = pd.read_csv(result_df_file, sep=",", low_memory=False)
# print(len(result_df))

result_df["rating"] = result_df["response"].astype(float)
# result_df["rating"] = result_df["rating_y"].astype(float)

# print(result_df)
if truncate_size_300:
    real_df = real_df[:300]
    result_df = result_df[:300]

latest_user_ratings = (real_df[real_df['userId'].isin(result_df)])

# 使用merge函数将两个DataFrame合并，并标记每个来源的数据
merged_df = pd.merge(real_df, result_df, on='userId', how='outer', suffixes=('_real_df', '_result_df'))

# 筛选出分数不同的项目
# print(merged_df)
different_ratings = merged_df[merged_df["rating_real_df"] != merged_df["rating_result_df"]]
different_ratings["diff"] = abs(different_ratings["rating_real_df"] - different_ratings["rating_result_df"])
# pd.set_option('display.max_rows', None)
print(different_ratings.sort_values(by="diff", ascending=False).head(50))
print("异常值:", (different_ratings["diff"] > 2.5).sum())


mse = mean_squared_error(real_df["rating"], result_df["rating"])
rmse = np.sqrt(mse)  # 计算RMSE，即MSE的平方根

mae = mean_absolute_error(real_df["rating"], result_df["rating"])

real_df.loc[real_df["rating"] >= 4, "tf"] = 1
real_df.loc[real_df["rating"] < 4, "tf"] = 0
result_df.loc[result_df["rating"] >= 4, "tf"] = 1
result_df.loc[result_df["rating"] < 4, "tf"] = 0

# true_labels = ((real_df["rating"] >= 4.0).round()).astype(int)


def mape(y_true, y_pred):
    return np.mean(np.abs((y_pred - y_true) / y_true)) * 100


def smape(y_true, y_pred):
    """
    计算对称平均绝对百分比误差（SMAPE）
    """
    denominator = (np.abs(y_true) + np.abs(y_pred)) / 2.0
    # 防止除零错误，当分子或分母为零时，设置最小可接受阈值（如1e-8）
    denominator = np.where(denominator == 0, 1e-8, denominator)

    absolute_percentage_errors = np.abs((y_true - y_pred) / denominator) * 100
    return np.mean(absolute_percentage_errors)

mape = mape(real_df["rating"], result_df["rating"])
smape = smape(real_df["rating"], result_df["rating"])

print("Recall:", recall_score(real_df["tf"], result_df["tf"]))
print("ROC-AUC:", roc_auc_score(real_df["tf"], result_df["tf"]))
print("RMSE:", rmse)
print("MAE:", mae)
print("MAPE:", mape)
print("SMAPE:", smape)
